package weblech.spider;

import java.util.*;
import java.net.URL;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;

public class Spider implements Runnable, Constants {
	
	private static final Logger logger = Logger.getLogger(Spider.class);	
	
    private SpiderConfig config;
    private DownloadQueue queue;
    
    /**   已下载URL的set   */
    private Set<URL> downloadedUrls;
    
    /** 当前正在爬的URL set */
    private Set<URLToDownload> urlsDownloading;
    
    /** 正在爬的链接的数量 */
    private int nowUrlsNum;
    
    /** 检查是否退出 */
    private boolean quit;
   
    /** 当前爬虫线程的数量 */
    private int runningThread;
    
    private int picCount;
    
    private int maxDepth;

    public Spider(SpiderConfig config) {
        this.config = config;
        queue = new DownloadQueue(config);
        queue.queueURL(new URLToDownload(config.getStartLocation(), 0));
        
        downloadedUrls = new HashSet<URL>();
        urlsDownloading = new HashSet<URLToDownload>();
        nowUrlsNum = 0;
        picCount = 0;
        maxDepth = config.getMaxDepth();
    }

    public void initialize() {
        quit = false;
        runningThread = 0;

        for(int i = 0; i < config.getSpiderThreads(); i++) {
            logger.info("Start Spider-Thread-" + (i + 1) + " ......");
            Thread t = new Thread(this, "Spider-Thread-" + (i + 1));
            t.start();
            runningThread++;
        }
    }
    
    public void run() {
        HTMLParser htmlParser = new HTMLParser();
        URLGetter urlGetter = new URLGetter(config);
        
        Random ran = new Random();
        while ((queueSize() > 0 || nowUrlsNum > 0) && !quit) {
            try {
                //给其它线程时间些准备时间
                Thread.sleep(ran.nextInt(500));
            } catch (InterruptedException e) {}
            
            if (queueSize() == 0 && nowUrlsNum > 0) {
                //下载队列为空，但是还有URL在下载中，所以此线程应该等待些时间
                try {
                    Thread.sleep(QUEUE_CHECK_INTERVAL);
                } catch (InterruptedException ignored) {}
                continue;
            } 
            if (queueSize() == 0 && nowUrlsNum == 0) {
                break;
            }
            
            URLToDownload currentURL = getNextUrl();
            if (currentURL == null) {
                break;
            }
            
            List<URL> newURLs = downloadURL(currentURL, urlGetter, htmlParser);
            
            int newDepth = currentURL.getDepth() + 1;
            newURLs = filterURLs(newURLs);
            
            logger.debug("newURLs : size = " + newURLs.size());
            
            ArrayList<URLToDownload> u2dsToQueue = new ArrayList<URLToDownload>();
            for(Iterator<URL> i = newURLs.iterator(); i.hasNext(); ) {
                URL u = i.next();
                synchronized(downloadedUrls) {
                    if(!downloadedUrls.contains(u) && (newDepth <= maxDepth)) {
                        u2dsToQueue.add(new URLToDownload(u, currentURL.getURL(), newDepth));
                        downloadedUrls.add(u);
                    }
                }
            }
            
            synchronized(queue) {
                queue.queueURLs(u2dsToQueue);
                nowUrlsNum--;
            }
        }
        
        runningThread--;
        
        if (runningThread == 0) {
            logger.info("image count : " + picCount);
        }
        
        logger.info("Spider thread " + Thread.currentThread().getName() + " stopping");
    }
    
    private URLToDownload getNextUrl() {
        URLToDownload nextURL = null;
        synchronized(queue) {
            nextURL = queue.getNextInQueue();
            if (nextURL != null) {
                nowUrlsNum++;
            } else {
                return null;
            }
        }
        
        synchronized(urlsDownloading) {
            urlsDownloading.add(nextURL);
        }
        
        return nextURL;
    }

    public void stop() {
        quit = true;
    }

    public boolean isRunning() {
        return runningThread != 0;
    }

    private int queueSize() {
        synchronized(queue) {
            return queue.size();
        }
    }

    /**
     *  starts to download
     */
    private List<URL> downloadURL(URLToDownload url, URLGetter urlGetter, HTMLParser htmlParser) {
        URLObject obj = urlGetter.getURL(url);
        
        if (obj != null) {
            if (obj.isImage()) {
                if (obj.getContentSize() < config.getImagesSizeLimit() * (1 << 10)) {
                    logger.info("image size less than initialize size : " + url.getURL().toExternalForm());
                    return new ArrayList<URL>();
                }
                
                obj.writeToFile();
                picCount++;
            } else if (obj.isXML() || obj.isHTML()) {
                boolean requestLink = false;
                if ((url.getDepth() + 1) < maxDepth) {
                    requestLink = true;
                }
                
                return htmlParser.parseLinksInDocument(obj.getStringContent(), requestLink);
            }
        }
        
        return new ArrayList<URL>();
    }

    /**
     * 过滤URL
     */
    private List<URL> filterURLs(List<URL> urls) {
        String match = config.getURLMatch();
        if (StringUtils.isBlank(match)) {
            return urls;
        }

        ArrayList<URL> retVal = new ArrayList<URL>();
        synchronized(downloadedUrls) {
            for(Iterator<URL> i = urls.iterator(); i.hasNext(); ) {
                URL u = i.next();
                if(downloadedUrls.contains(u)) {
                    continue;
                }
                
                String s = u.toExternalForm();
                if(s.indexOf(match) != -1) {
                    retVal.add(u);
                }
            }
        }
        
        return retVal;
    }
    
}
